Recipe Generator

Work Distribution

Diane

dmp282@cornell.edu

Did the programming, ideation and testing

Koushani

kd496@cornell.edu

Experimented with detection methods, assisted with testing, acquired a 3D-printed frame.

Code Appendix

recipeSelector.py

// recipeSelector.py
from display import openRecipe, cameraMode, processing, waitForButton
from camTest import camera
import numpy as np
#import RPi.GPIO as GPIO

ingredients=np.array([])
loop=True


while(loop):
        # Take photos to identify ingredients
        cameraMode()
        # Process to get ingredient name
        ingred=processing()
        print(ingred)
        if ingred != "not found":
                if ingredients.size > 0:
                        ingredients=np.append(ingredients, ingred)
                else:
                        ingredients=np.array([ingred])
                print(ingredients)
        loop=waitForButton()

# Display recipe fetched from SQL database
if ingredients.size == 0:
        ingredients = np.array(["not found"])
openRecipe(ingredients)
}
              

database.py

//database.py
from mysql.connector import connect, Error
# not used: test function to see the databases my user has access to
def displayDatabases():
        # log into database
        try:
                with connect(
                host="localhost",
                user='pi',
                password='pi',
                ) as connection:
                        print(connection)
                        # display databases available/created
                        with connection.cursor() as cursor:
                                cursor.execute("SHOW DATABASES")
                                for db in cursor:
                                        print(db)

        except Error as e:
                print(e)

# Not used: test to see all recipes in SQL database "finalProject" "recipes" table
def displayRecipeTable():
        try:
                with connect(
                host="localhost",
                user='pi',
                password='pi',
                database='finalProject',
                ) as connection:
                        print(connection)
                        # display table
                        with connection.cursor() as cursor:
                                cursor.execute("SELECT * FROM recipes LIMIT 50")
                                result=cursor.fetchall()
                                for row in result:
                                        print(row)

        except Error as e:
                print(e)


# return pdf with title String nameRec
def findRecipe(nameRec):
        try:
                with connect(
                host="localhost",
                user='pi',
                password='pi',
                database='finalProject',
                ) as connection:
                        print(connection)
                        # display table
                        with connection.cursor() as cursor:
                                cursor.execute("SELECT * FROM recipes WHERE name LIKE '"+nameRec+"'")
                                result=cursor.fetchall()
                                for row in result:
                                        return(row)
        except Error as e:
                print(e)

# return pdf with ingredients specified by ingredient
def findRecipeByIngredient(ingredients):
        try:
                with connect(
                host="localhost",
                user='pi',
                password='pi',
                database='finalProject',
                ) as connection:
                        print(connection)
                        # display table
                        with connection.cursor() as cursor:
                                cursor.execute("SELECT * FROM recipes WHERE ingredients LIKE '%" + ingredients + "%'")
                                result=cursor.fetchall()
                                for row in result:
                                        return(row)
        except Error as e:
                print(e)
        

        

display.py

 //display.py
import webbrowser
from database import findRecipeByIngredient, displayRecipeTable, findRecipe
import pygame
import os
import RPi.GPIO as GPIO
import time
import cv2
from camTest import camera, barcode_image, preProcess
import subprocess
import numpy as np
#from camera import textID

pygame.init()

currStep = 0
takePicture= False
loop = True

#setup piTFT
os.putenv('SDL_VIDEODRIVER', 'fbcon')
os.putenv('SDL_FBDEV', '/dev/fb1')
os.putenv('SDL_MOUSEDRV', 'dummy')
os.putenv('SDL_MOUSEDEV', '/dev/null')
os.putenv('DISPLAY', '')

pygame.mouse.set_visible(False)

#setup next(27), photo (23) and quit(17) buttons
GPIO.setmode(GPIO.BCM)
GPIO.setup(17, GPIO.IN, pull_up_down=GPIO.PUD_UP)
GPIO.setup(27, GPIO.IN, pull_up_down=GPIO.PUD_UP)
GPIO.setup(23, GPIO.IN, pull_up_down=GPIO.PUD_UP)
GPIO.setup(22, GPIO.IN, pull_up_down=GPIO.PUD_UP)

def GPIO22_callback(channel):
        global loop
        loop = False

size = width, height = 320, 240
screen = pygame.display.set_mode(size)
font = pygame.font.Font(None, 25)
black = 0,0,0
cameraMode = False

def GPIO17_callback(channel):
                GPIO.cleanup()
                pygame.quit()

def GPIO23_callback(channel):
        if cameraMode:
                global takePicture
                takePicture= True
                camera('pic')
                print("took picture!")
#picture button
GPIO.add_event_detect(17, GPIO.FALLING, callback=GPIO17_callback)
GPIO.add_event_detect(23, GPIO.FALLING, callback=GPIO23_callback)
GPIO.add_event_detect(22, GPIO.FALLING, callback=GPIO22_callback)


def openRecipe(ingredients):
        notFound=False
        for i in range(ingredients.size):
                if ingredients[i] == "not found":
                        notFound=True
        if notFound:
                screen.fill(black)
                text_surface = font.render("No Ingredients input", True, (255, 255, 255))
                screen.blit(text_surface, (0,0))
                pygame.display.flip()
                GPIO.cleanup()
                return
       
       # find recipe based on ingredients
        numIngreds=np.array([["",0]])
        for ingred in ingredients:
                #print(ingred)
                name,recIngreds,path,maxSteps=findRecipeByIngredient(ingred)
                #print(name)
                contained=False
                for i in range(np.shape(numIngreds)[0]):
                        print(numIngreds[i,0])
                        if  name == numIngreds[i,0]:
                                numIngreds[1,i] = int(numIngreds[i,1])+1
                                contained=True
                if contained == False:
                        numIngreds = np.append(numIngreds, [[name,1]], axis=0)

        print(numIngreds)

        mostName=""
        mostIngreds=0
        for i in range(np.shape(numIngreds)[0]):
                if int(numIngreds[i,1]) > mostIngreds:
                        mostName=numIngreds[i,0]
                        
        #fetch from database
        name,recIngred,path,maxSteps=findRecipe(mostName)

        def GPIO27_callback(channel):
                #next step in recipe
                global currStep
                if currStep<(int(maxSteps)+1):
                        currStep +=1
                else:
                        pygame.quit()
                time.sleep(2)
        GPIO.add_event_detect(27, GPIO.FALLING, callback=GPIO27_callback)

        progress=True
        while (progress):
                if currStep==0:
                        img=pygame.image.load(path+'/'+name+'.png')
                elif currStep < (maxSteps+1):
                        img=pygame.image.load(path+'/step'+str(currStep)+'.png')
                else:
                        progress=False
                screen.fill(black)
                resized = pygame.transform.scale(img,(320,img.get_rect().height));
                screen.blit(resized, (0,0))
                pygame.display.flip()

        GPIO.cleanup()



def cameraMode():
        global cameraMode
        cameraMode = True
        global takePicture
        progress=True
        while (progress):
                time.sleep(0.1)
                print("camMode")
                if takePicture:
                        print("done")
                        img=pygame.image.load('capture.jpg')
                        screen.fill(black)
                        resized = pygame.transform.scale(img,(320,240));
                        screen.blit(resized, (0,0))
                        pygame.display.flip()
                        progress=False
                        cameraMode=False
                        takePicture=False
                else:
                        #print("cameraOn")
                        camera('display')
                        img=pygame.image.load('capture.jpg')
                        screen.fill(black)
                        resized = pygame.transform.scale(img,(320,240));
                        screen.blit(resized, (0,50))
                        text_surface = font.render("Press Button 23 to take a Photo", True, (255, 255, 255))
                        screen.blit(text_surface, (10, 30))
                        pygame.display.flip()

        time.sleep(3)


# preprocess, detect barcode, decode to ingredient name and return
def processing():
        print("process")
        preProcess()
        img,detected,text=barcode_image()
        #map barcode to ingredient
        if text=='4099100231953':
                text="linguine"
        elif text =="6009001014119" :
                text="pepper"
        else:
                text="not found"
        
        screen.fill(black)
        text_surface = font.render(str(text), True, (255, 255, 255))
        screen.blit(text_surface, (10, 30))
        pygame.display.flip()
        time.sleep(5)
        #print(text)
        return text

# Delays operation to allow user time to give input indicating all barcodes have been scanned
def waitForButton():
        global loop
        screen.fill(black)
        text_surface = font.render("Press 22 to stop camera", True, (255, 255, 255))
        screen.blit(text_surface, (10, 30))
        pygame.display.flip()
        time.sleep(5)
        return loop



        

camTest.py

//camtest.py
import cv2
import threading
from pyzbar import pyzbar
import numpy as np


command = None
cap = cv2.VideoCapture(0)
reqCommand = 'pic'

#Function to take a photo
def camera():
    while(True):
        ret, frame = cap.read()

        if ret:
            out = cv2.imwrite('capture.jpg', frame)
            #print('success')
            break


#Function to decode a barcode (adapted from Smart Fridge project)
def barcode_image():
    img = cv2.imread('capture.jpg')
    barcodes = pyzbar.decode(img)
    barcode_detected = False
    barcode_text = ''
    for barcode in barcodes:
        barcode_detected = True
        barcode_text = barcode.data.decode('utf-8')
        barcode_type = barcode.type
        print(f"Barcode: {barcode_text} )
    if barcode_detected == False:
        barcode_text="not detected"

    print(barcode_text)
    return img, barcode_detected, barcode_text

#Convert image to grayscale
def preProcess():
    img=cv2.imread("capture.jpg")
    img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    out = cv2.imwrite('capture.jpg', img_gray)


#Test function for debugging the picamera
def tester():
    from picamera import PiCamera
    from time import sleep

    camera = PiCamera()

    camera.start_preview()
    sleep(100)

    camera.stop_preview()